home *** CD-ROM | disk | FTP | other *** search
/ Internet Magazine 2003 Spring / INTERNET101.ISO / pc / software / windows / building / robot_manager / robotmanagersetup.exe / {app} / Sample.rbt < prev    next >
Encoding:
Extensible Markup Language  |  2002-12-04  |  15.3 KB  |  289 lines
  1. <?xml version="1.0" encoding="UTF-8"?>
  2. <robot_Manager steps="195">
  3.   <web_site use_ftp="0">
  4.     <loc_local>
  5.       <path>D:\_Applications_\Robot-Manager\Sample</path>
  6.       <compression>0</compression>
  7.     </loc_local>
  8.     <loc_ftp>
  9.       <server></server>
  10.       <port>21</port>
  11.       <user_name></user_name>
  12.       <password></password>
  13.       <passive>0</passive>
  14.       <path></path>
  15.       <compression>0</compression>
  16.     </loc_ftp>
  17.   </web_site>
  18.   <access_log capture_method="local">
  19.     <loc_local>
  20.       <path>D:\_Applications_\Robot-Manager\Web.log</path>
  21.       <compression>0</compression>
  22.     </loc_local>
  23.     <loc_ftp>
  24.       <server></server>
  25.       <port>21</port>
  26.       <user_name></user_name>
  27.       <password></password>
  28.       <passive>0</passive>
  29.       <path></path>
  30.       <compression>0</compression>
  31.     </loc_ftp>
  32.     <loc_telnet>
  33.       <host></host>
  34.       <port>23</port>
  35.       <user_name></user_name>
  36.       <password></password>
  37.       <path></path>
  38.       <compressed>0</compressed>
  39.       <prompts>
  40.         <login>ogin:</login>
  41.         <password>assword:</password>
  42.         <command>]$</command>
  43.       </prompts>
  44.       <commands>
  45.         <robots>grep -i '/robots.txt' %PATH%</robots>
  46.         <pages>grep '%ROBOT%' %PATH%</pages>
  47.         <robots_comp>gunzip -c %PATH% | grep -i '/robots.txt'</robots_comp>
  48.         <pages_comp>gunzip -c %PATH% | grep '%ROBOT%'</pages_comp>
  49.       </commands>
  50.     </loc_telnet>
  51.     <spider_logs>
  52.       <parsed>1</parsed>
  53.       <entry_size>9</entry_size>
  54.       <date_index>3</date_index>
  55.       <spider_index>8</spider_index>
  56.       <page_index>4</page_index>
  57.       <remote_ip_index>0</remote_ip_index>
  58.       <error_code_index>5</error_code_index>
  59.       <spider_type>user-agent</spider_type>
  60.       <page_type>request-line</page_type>
  61.       <date_type>apache</date_type>
  62.       <epressions>
  63.         <apache_date>^(?:31|[123]0|[012]?[1-9])[-/][A-Z][a-z]{2}[-/](?:19|20)(?:[0-9]{2})?:(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\s[-+\s][0-9]{1,4}$</apache_date>
  64.         <iis_date>^(?:19|20)[0-9]{2}[-/](?:0[1-9]|1[012])[-/](?:31|[123]0|[012]?[1-9])$</iis_date>
  65.         <us_euro_date>^(?:31|[123]0|[012]?[1-9])[-/](?:0[1-9]|1[012])[-/](?:19|20)[0-9]{2}$</us_euro_date>
  66.         <user_agent>^Mozilla/[0-9]\.[0-9].*$</user_agent>
  67.         <remote_host>^[-a-zA-Z]+(?:\.[-a-zA-Z0-9]+)*\.(?:com|edu|gov|int|mil|net|org|biz|info|name|museum|coop|aero|[a-zA-Z][a-zA-Z])$</remote_host>
  68.         <remote_ip>^(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-5][0-5])\.(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-5][0-5])\.(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-5][0-5])\.(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-5][0-5])$</remote_ip>
  69.         <request_line>^(?:GET|POST|HEAD)\s/.*\sHTTP/1\.[01]$</request_line>
  70.         <filename>^/.*</filename>
  71.         <error_code>(?:^10[01]$)|(?:^20[0-6]$)|(?:^30[0-7]$)|(?:^4(?:0[0-9]|1[0-7])$)|(?:^50[0-5]$)</error_code>
  72.       </epressions>
  73.       <logs>
  74.       </logs>
  75.     </spider_logs>
  76.   </access_log>
  77.   <paths>
  78.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\About\"/>
  79.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\About\French\"/>
  80.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\About\French\about.html"/>
  81.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\About\French\vssver.scc"/>
  82.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\About\German\"/>
  83.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\About\German\about.html"/>
  84.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\About\German\vssver.scc"/>
  85.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\About\about.html"/>
  86.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\About\vssver.scc"/>
  87.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\Contact\"/>
  88.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\Contact\French\"/>
  89.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Contact\French\contact.html"/>
  90.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Contact\French\vssver.scc"/>
  91.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\Contact\German\"/>
  92.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Contact\German\contact.html"/>
  93.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Contact\German\vssver.scc"/>
  94.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Contact\contact.html"/>
  95.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Contact\vssver.scc"/>
  96.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\Faq\"/>
  97.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Faq\faq.html"/>
  98.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Faq\vssver.scc"/>
  99.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\"/>
  100.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\FakeImage01.gif"/>
  101.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\FakeImage02.gif"/>
  102.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\FakeImage03.gif"/>
  103.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\FakeImage04.gif"/>
  104.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\French\"/>
  105.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\French\FakeImage01.gif"/>
  106.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\French\FakeImage02.gif"/>
  107.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\French\FakeImage03.gif"/>
  108.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\French\FakeImage04.gif"/>
  109.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\French\vssver.scc"/>
  110.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\German\"/>
  111.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\German\FakeImage01.gif"/>
  112.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\German\FakeImage02.gif"/>
  113.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\German\FakeImage03.gif"/>
  114.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\German\FakeImage04.gif"/>
  115.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\German\vssver.scc"/>
  116.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Images\vssver.scc"/>
  117.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\Products\"/>
  118.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Products\order-page.html"/>
  119.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Products\product01.html"/>
  120.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Products\product02.html"/>
  121.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Products\vssver.scc"/>
  122.     <path googlebot="" item_type="1" robots="" value="D:\_Applications_\Robot-Manager\Sample\Resource\"/>
  123.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Resource\affiliation.html"/>
  124.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Resource\publication.html"/>
  125.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Resource\resource.html"/>
  126.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\Resource\vssver.scc"/>
  127.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\index.html"/>
  128.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\robots.txt"/>
  129.     <path googlebot="" item_type="16" robots="" value="D:\_Applications_\Robot-Manager\Sample\vssver.scc"/>
  130.   </paths>
  131.   <spiders>
  132.     <spider disallow="0" name="ADSA Research Group" parent="" selected="0">
  133.     </spider>
  134.     <spider disallow="0" name="Abacho" parent="" selected="0">
  135.     </spider>
  136.     <spider disallow="0" name="Acoon" parent="" selected="1">
  137.       <disallow path="/About/French/"/>
  138.       <disallow path="/About/about.html"/>
  139.       <disallow path="/Contact/French/"/>
  140.       <disallow path="/Contact/contact.html"/>
  141.       <disallow path="/Faq/faq.html"/>
  142.       <disallow path="/Images/"/>
  143.       <disallow path="/Products/order-page.html"/>
  144.     </spider>
  145.     <spider disallow="0" name="All Spiders" parent="" selected="1">
  146.       <disallow path="/About/"/>
  147.       <disallow path="/Contact/"/>
  148.       <disallow path="/Faq/"/>
  149.       <disallow path="/Images/"/>
  150.       <disallow path="/Products/order-page.html"/>
  151.       <disallow path="/Resource/"/>
  152.     </spider>
  153.     <spider disallow="0" name="AllThatNet" parent="" selected="0">
  154.     </spider>
  155.     <spider disallow="0" name="Alta Vista" parent="" selected="0">
  156.     </spider>
  157.     <spider disallow="0" name="Anzwers" parent="" selected="0">
  158.     </spider>
  159.     <spider disallow="0" name="AustLII" parent="" selected="0">
  160.     </spider>
  161.     <spider disallow="0" name="Business.com" parent="" selected="0">
  162.     </spider>
  163.     <spider disallow="0" name="CMC/0.01" parent="e-collector" selected="1">
  164.     </spider>
  165.     <spider disallow="0" name="DeepIndex" parent="" selected="0">
  166.     </spider>
  167.     <spider disallow="0" name="Direct Hit Grabber" parent="" selected="0">
  168.     </spider>
  169.     <spider disallow="0" name="Entire Web" parent="" selected="0">
  170.     </spider>
  171.     <spider disallow="0" name="EuroSeek" parent="" selected="0">
  172.     </spider>
  173.     <spider disallow="0" name="Excite" parent="" selected="0">
  174.     </spider>
  175.     <spider disallow="0" name="FAST/AllTheWeb" parent="" selected="0">
  176.     </spider>
  177.     <spider disallow="0" name="Fireball" parent="" selected="0">
  178.     </spider>
  179.     <spider disallow="0" name="GenDoor" parent="" selected="0">
  180.     </spider>
  181.     <spider disallow="0" name="Gigabot" parent="" selected="0">
  182.     </spider>
  183.     <spider disallow="0" name="Goo" parent="" selected="0">
  184.     </spider>
  185.     <spider disallow="0" name="Google" parent="" selected="0">
  186.     </spider>
  187.     <spider disallow="0" name="Google Image" parent="e-collector" selected="1">
  188.     </spider>
  189.     <spider disallow="0" name="Griffon" parent="" selected="0">
  190.     </spider>
  191.     <spider disallow="0" name="Hämähäkki" parent="" selected="0">
  192.     </spider>
  193.     <spider disallow="0" name="Industry Central" parent="" selected="0">
  194.     </spider>
  195.     <spider disallow="0" name="InfoSeek.de" parent="" selected="0">
  196.     </spider>
  197.     <spider disallow="0" name="Ingrid" parent="" selected="0">
  198.     </spider>
  199.     <spider disallow="0" name="Inktomi" parent="" selected="0">
  200.     </spider>
  201.     <spider disallow="0" name="Innerprise" parent="" selected="0">
  202.     </spider>
  203.     <spider disallow="0" name="Internet Cruiser" parent="" selected="0">
  204.     </spider>
  205.     <spider disallow="0" name="Kvasir" parent="" selected="0">
  206.     </spider>
  207.     <spider disallow="0" name="Legs" parent="" selected="0">
  208.     </spider>
  209.     <spider disallow="0" name="Lets Find It Now!" parent="" selected="0">
  210.     </spider>
  211.     <spider disallow="0" name="LinkAlarm" parent="" selected="0">
  212.     </spider>
  213.     <spider disallow="0" name="Lisa" parent="" selected="0">
  214.     </spider>
  215.     <spider disallow="0" name="Lycos" parent="" selected="0">
  216.     </spider>
  217.     <spider disallow="0" name="MPRM Group Limited" parent="" selected="0">
  218.     </spider>
  219.     <spider disallow="0" name="Mirago" parent="" selected="0">
  220.     </spider>
  221.     <spider disallow="0" name="NetMechanic" parent="" selected="0">
  222.     </spider>
  223.     <spider disallow="0" name="NetScoop" parent="" selected="0">
  224.     </spider>
  225.     <spider disallow="0" name="Northern Light" parent="" selected="0">
  226.     </spider>
  227.     <spider disallow="0" name="ODiN" parent="" selected="0">
  228.     </spider>
  229.     <spider disallow="0" name="OMEGA Find" parent="" selected="0">
  230.     </spider>
  231.     <spider disallow="0" name="Openfind" parent="" selected="0">
  232.     </spider>
  233.     <spider disallow="0" name="PlanetSearch" parent="" selected="0">
  234.     </spider>
  235.     <spider disallow="0" name="PolyBOT" parent="" selected="0">
  236.     </spider>
  237.     <spider disallow="0" name="Pompos" parent="" selected="0">
  238.     </spider>
  239.     <spider disallow="0" name="Portal Juice" parent="" selected="0">
  240.     </spider>
  241.     <spider disallow="0" name="RUTGERS" parent="" selected="0">
  242.     </spider>
  243.     <spider disallow="0" name="ScoutAbout" parent="" selected="0">
  244.     </spider>
  245.     <spider disallow="0" name="SearchUK" parent="" selected="0">
  246.     </spider>
  247.     <spider disallow="0" name="Seven Twentyfour" parent="" selected="0">
  248.     </spider>
  249.     <spider disallow="0" name="Speedy" parent="" selected="0">
  250.     </spider>
  251.     <spider disallow="0" name="Suchmaschine21" parent="" selected="0">
  252.     </spider>
  253.     <spider disallow="0" name="Thunderstone" parent="" selected="0">
  254.     </spider>
  255.     <spider disallow="0" name="TopicLink" parent="" selected="0">
  256.     </spider>
  257.     <spider disallow="0" name="TurnitinBOT" parent="" selected="0">
  258.     </spider>
  259.     <spider disallow="0" name="VietGATE" parent="" selected="0">
  260.     </spider>
  261.     <spider disallow="0" name="Wayback Machine" parent="" selected="0">
  262.     </spider>
  263.     <spider disallow="0" name="WhoWhere?" parent="" selected="0">
  264.     </spider>
  265.     <spider disallow="0" name="WholeWeb" parent="" selected="0">
  266.     </spider>
  267.     <spider disallow="0" name="WiseGuys" parent="" selected="0">
  268.     </spider>
  269.     <spider disallow="0" name="WiseNut" parent="" selected="0">
  270.     </spider>
  271.     <spider disallow="0" name="Zeus" parent="" selected="0">
  272.     </spider>
  273.     <spider disallow="1" name="e-collector" parent="" selected="1">
  274.     </spider>
  275.     <spider disallow="0" name="iaNett.com" parent="" selected="0">
  276.     </spider>
  277.     <spider disallow="0" name="kensaku.jp" parent="" selected="0">
  278.     </spider>
  279.     <spider disallow="0" name="mopilot.com" parent="" selected="0">
  280.     </spider>
  281.     <spider disallow="0" name="nathan" parent="" selected="0">
  282.     </spider>
  283.     <spider disallow="0" name="newscan" parent="" selected="0">
  284.     </spider>
  285.     <spider disallow="0" name="whatUseek" parent="" selected="0">
  286.     </spider>
  287.   </spiders>
  288. </robot_Manager>
  289.